/**
 * 
 */
package ca.uwindsor.cs.deepweb.utility.wikipedia;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.w3c.dom.Document;
import org.w3c.dom.Element;

/**
 * The input file is a txt file that is generated by a Tool called
 * WikiToMDict_2.0.exe, it is a converter from wiki XML dump file to TXT file.
 * All title and corresponding content have been extracted from the XML file.
 * But the txt file has its format and useless information. All we need in the
 * experiment are the titles and contents. The converter will write clean the
 * file and write to a new XML with defined tag:
 * 
 * @author Liang Jie
 * 
 */
public class WikiToXMLConverter {

	/**
	 * Instance of buffered reader to read the data source file
	 */
	protected BufferedReader bufferedreader_in;

	/**
	 * The content of a title
	 */
	protected String string_content;

	/**
	 * The title in zh_Wikipedia
	 */
	protected String string_title;

	/**
	 * XML�ļ��ı�����
	 */
	private Document document;

	public WikiToXMLConverter() {
		InputStreamReader read = null;
		try {
			read = new InputStreamReader(new FileInputStream(
					"SourceData//wiki.txt"), "UTF-8");
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		bufferedreader_in = new BufferedReader(read);

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		DocumentBuilder builder;
		try {
			builder = factory.newDocumentBuilder();
			document = builder.newDocument();
		} catch (ParserConfigurationException ex) {
			ex.printStackTrace();
		}

	}

	protected void filter() {
		char[] chars = string_content.toCharArray();
		StringBuffer temp = new StringBuffer();
		for (int offset_1 = 0; offset_1 < chars.length; offset_1++) {
			if (chars[offset_1] == '<') {
				for (int offset_2 = offset_1 + 1; offset_2 < chars.length; offset_2++) {
					if (chars[offset_2] == '>') {
						offset_1 = offset_2;
						break;
					}
				}
			} else {
				temp.append(chars[offset_1]);
			}
		}

		Pattern pa = Pattern
				.compile("[\u4e00-\u9fa5]|[0-9]|[��]|[��]|[��]|[��]|[��]|[��]|[��]|[��]|[��]");
		Matcher ma = pa.matcher(temp.toString());
		StringBuffer outputbuffer = new StringBuffer();
		while (ma.find()) {
			outputbuffer.append(ma.group());
		}
		this.string_content = outputbuffer.toString();
	}

	/**
	 * ��ʵ���ı����浽�ļ�
	 */
	private void saveToFile() {

		try {
			TransformerFactory tf = TransformerFactory.newInstance();
			Transformer transformer = tf.newTransformer();
			DOMSource source = new DOMSource(document);
			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");// ��������
			transformer.setOutputProperty(OutputKeys.INDENT, "YES");// �Ƿ�����
			FileOutputStream pw = new FileOutputStream("SourceData//wiki_temp"
					+ ".xml");
			StreamResult result = new StreamResult(pw);
			transformer.transform(source, result);
			pw.close();
		} catch (TransformerException mye) {
			mye.printStackTrace();
		} catch (IOException exp) {
			exp.printStackTrace();
		}
	}

	public void go2() {
		Element root = document.createElement("zh_wikipedia");
		document.appendChild(root);

		String string_line = new String();
		Vector<String> v = new Vector<String>();
		try {
			string_line = bufferedreader_in.readLine();
			while (string_line != null) {
				if (string_line.startsWith("</>")) {

					Element node = document.createElement("node");

					if (v.size() > 2) {
						string_title = v.get(0);
						string_content = v.get(1);
						StringBuffer bu = new StringBuffer();
						for (int offset = 2; offset < v.size(); offset++) {

							bu.append(v.get(offset));
						}
						string_content += bu.toString();
					} else {
						string_title = v.get(0);
						string_content = v.get(1);
					}
					filter();
					node.setAttribute("content", string_content);
					node.setAttribute("title", string_title);
					//node.appendChild(document.createTextNode(string_content));
					
					root.appendChild(node);

					v.clear();
					string_line = bufferedreader_in.readLine();
				} else {
					v.add(string_line);
					string_line = bufferedreader_in.readLine();
				}
			}

		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		
		System.out.println("Convertion complete!");
		saveToFile();
	}

	/**
	 * @param args
	 *            the command line arguments
	 */
	public static void main(String[] args) {
		WikiToXMLConverter con = new WikiToXMLConverter();
		con.go2();
	}
}
